# Load data from huggingface
from datasets import load_dataset
dataset = load_dataset("maartengr/arxiv_nlp")["train"]
# Extract metadata
abstracts = list(dataset["Abstracts"])
titles = list(dataset["Titles"])Chapter 5 - Text Clustering and Topic Modeling
ArXiv Articles: Computation and Language
A Common Pipeline for Text Clustering
1. Embedding Documents
from sentence_transformers import SentenceTransformer
# Create an embedding for each abstract
embedding_model = SentenceTransformer('thenlper/gte-small')
embeddings = embedding_model.encode(abstracts, show_progress_bar=True)# Check the dimensions of the resulting embeddings
embeddings.shape(44949, 384)2. Reducing the Dimensionality of Embeddings
from umap import UMAP
# We reduce the input embeddings from 384 dimenions to 5 dimenions
umap_model = UMAP(
n_components=5, min_dist=0.0, metric='cosine', random_state=42
)
reduced_embeddings = umap_model.fit_transform(embeddings)/home/alal/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
warn(3. Cluster the Reduced Embeddings
from hdbscan import HDBSCAN# We fit the model and extract the clusters
hdbscan_model = HDBSCAN(
min_cluster_size=50, metric='euclidean', cluster_selection_method='eom'
).fit(reduced_embeddings)
clusters = hdbscan_model.labels_
# How many clusters did we generate?
len(set(clusters))164Inspecting the Clusters
Manually inspect the first three documents in cluster 0:
import numpy as np
# Print first three documents in cluster 0
cluster = 0
for index in np.where(clusters==cluster)[0][:3]:
print(abstracts[index][:300] + "... \n") This works aims to design a statistical machine translation from English text
to American Sign Language (ASL). The system is based on Moses tool with some
modifications and the results are synthesized through a 3D avatar for
interpretation. First, we translate the input text to gloss, a written fo...
Researches on signed languages still strongly dissociate lin- guistic issues
related on phonological and phonetic aspects, and gesture studies for
recognition and synthesis purposes. This paper focuses on the imbrication of
motion and meaning for the analysis, synthesis and evaluation of sign lang...
Modern computational linguistic software cannot produce important aspects of
sign language translation. Using some researches we deduce that the majority of
automatic sign language translation systems ignore many aspects when they
generate animation; therefore the interpretation lost the truth inf...
Next, we reduce our embeddings to 2-dimensions so that we can plot them and get a rough understanding of the generated clusters.
import pandas as pd
# Reduce 384-dimensional embeddings to 2 dimensions for easier visualization
reduced_embeddings = UMAP(
n_components=2, min_dist=0.0, metric='cosine', random_state=42
).fit_transform(embeddings)
# Create dataframe
df = pd.DataFrame(reduced_embeddings, columns=["x", "y"])
df["title"] = titles
df["cluster"] = [str(c) for c in clusters]
# Select outliers and non-outliers (clusters)
clusters_df = df.loc[df.cluster != "-1", :]
outliers_df = df.loc[df.cluster == "-1", :]/home/alal/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
warn(Static Plot
import matplotlib.pyplot as plt
# Plot outliers and non-outliers seperately
plt.scatter(outliers_df.x, outliers_df.y, alpha=0.05, s=2, c="grey")
plt.scatter(
clusters_df.x, clusters_df.y, c=clusters_df.cluster.astype(int),
alpha=0.6, s=2, cmap='tab20b'
)
plt.axis('off')
# plt.savefig("matplotlib.png", dpi=300) # Uncomment to save the graph as a .png(np.float64(-7.382833409309387),
np.float64(10.984574723243714),
np.float64(-2.5725981533527373),
np.float64(16.283057743310927))
From Text Clustering to Topic Modeling
BERTopic: A Modular Topic Modeling Framework
from bertopic import BERTopic
# Train our model with our previously defined models
topic_model = BERTopic(
embedding_model=embedding_model,
umap_model=umap_model,
hdbscan_model=hdbscan_model,
verbose=True
).fit(abstracts, embeddings)2026-01-19 21:47:57,272 - BERTopic - Dimensionality - Fitting the dimensionality reduction algorithm
2026-01-19 21:48:21,113 - BERTopic - Dimensionality - Completed ✓
2026-01-19 21:48:21,114 - BERTopic - Cluster - Start clustering the reduced embeddings
2026-01-19 21:48:21,828 - BERTopic - Cluster - Completed ✓
2026-01-19 21:48:21,833 - BERTopic - Representation - Fine-tuning topics using representation models.
2026-01-19 21:48:23,635 - BERTopic - Representation - Completed ✓Now, let’s start exploring the topics that we got by running the code above.
topic_model.get_topic_info()| Topic | Count | Name | Representation | Representative_Docs | |
|---|---|---|---|---|---|
| 0 | -1 | 15711 | -1_the_of_and_to | [the, of, and, to, in, we, for, language, that... | [ Few-shot question answering (QA) aims at ac... |
| 1 | 0 | 2055 | 0_speech_asr_recognition_end | [speech, asr, recognition, end, acoustic, audi... | [ All-neural, end-to-end ASR systems gained r... |
| 2 | 1 | 1089 | 1_translation_nmt_machine_bleu | [translation, nmt, machine, bleu, neural, engl... | [ Neural machine translation (NMT) becomes a ... |
| 3 | 2 | 853 | 2_summarization_summaries_summary_abstractive | [summarization, summaries, summary, abstractiv... | [ Evaluation of text summarization approaches... |
| 4 | 3 | 829 | 3_hate_offensive_speech_detection | [hate, offensive, speech, detection, toxic, so... | [ The goal of hate speech detection is to fil... |
| ... | ... | ... | ... | ... | ... |
| 159 | 158 | 53 | 158_spelling_csc_correction_chinese | [spelling, csc, correction, chinese, errors, e... | [ Chinese Spell Checking (CSC) task aims to d... |
| 160 | 159 | 53 | 159_counseling_mental_therapy_health | [counseling, mental, therapy, health, psychoth... | [ Mental health care poses an increasingly se... |
| 161 | 160 | 52 | 160_reviews_opinion_summaries_summarization | [reviews, opinion, summaries, summarization, r... | [ When faced with a large number of product r... |
| 162 | 161 | 52 | 161_gans_gan_adversarial_generation | [gans, gan, adversarial, generation, generativ... | [ Text generation is of particular interest i... |
| 163 | 162 | 52 | 162_backdoor_attacks_attack_triggers | [backdoor, attacks, attack, triggers, poisoned... | [ Deep neural networks (DNNs) and natural lan... |
164 rows × 5 columns
Hundreds of topics were generated using the default model! To get the top 10 keywords per topic as well as their c-TF-IDF weights, we can use the get_topic() function:
topic_model.get_topic(0)[('speech', np.float64(0.028044048463799758)),
('asr', np.float64(0.019103557225532256)),
('recognition', np.float64(0.013552091477301144)),
('end', np.float64(0.010289302278892161)),
('acoustic', np.float64(0.00973514556960071)),
('audio', np.float64(0.006846988836039268)),
('speaker', np.float64(0.006746370403108714)),
('wer', np.float64(0.0065942814733770585)),
('error', np.float64(0.006503600337995727)),
('automatic', np.float64(0.006073033426036759))]We can use the find_topics() function to search for specific topics based on a search term. Let’s search for a topic about topic modeling:
topic_model.find_topics("topic modeling")([25, -1, 43, 39, 88],
[np.float32(0.9548093),
np.float32(0.9129914),
np.float32(0.9078753),
np.float32(0.90433663),
np.float32(0.9037678)])It returns that topic 22 has a relatively high similarity (0.95) with our search term. If we then inspect the topic, we can see that it is indeed a topic about topic modeling:
topic_model.get_topic(25)[('topic', np.float64(0.06762386852487184)),
('topics', np.float64(0.035761575158132886)),
('lda', np.float64(0.016942399804845947)),
('latent', np.float64(0.013364966654403065)),
('documents', np.float64(0.012795304337419438)),
('document', np.float64(0.012745519708000716)),
('modeling', np.float64(0.012114985329208914)),
('dirichlet', np.float64(0.010212517055996743)),
('word', np.float64(0.008675659463331778)),
('allocation', np.float64(0.007980208492860665))]That seems like a topic that is, in part, characterized by the classic LDA technique. Let’s see if the BERTopic paper was also assigned to topic 22:
topic_model.topics_[titles.index('BERTopic: Neural topic modeling with a class-based TF-IDF procedure')]25It is! We expected it might be because there are non-LDA specific words in the topic describtion such as “clustering” and “topic”.
Visualizations
Visualize Documents
# Visualize topics and documents
fig = topic_model.visualize_documents(
titles,
reduced_embeddings=reduced_embeddings,
width=1200,
hide_annotations=True
)
# Update fonts of legend for easier visualization
fig.update_layout(font=dict(size=16))--------------------------------------------------------------------------- ValueError Traceback (most recent call last) File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/IPython/core/formatters.py:984, in IPythonDisplayFormatter.__call__(self, obj) 982 method = get_real_method(obj, self.print_method) 983 if method is not None: --> 984 method() 985 return True File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/plotly/basedatatypes.py:850, in BaseFigure._ipython_display_(self) 847 import plotly.io as pio 849 if pio.renderers.render_on_display and pio.renderers.default: --> 850 pio.show(self) 851 else: 852 print(repr(self)) File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/plotly/io/_renderers.py:415, in show(fig, renderer, validate, **kwargs) 410 raise ValueError( 411 "Mime type rendering requires ipython but it is not installed" 412 ) 414 if not nbformat or Version(nbformat.__version__) < Version("4.2.0"): --> 415 raise ValueError( 416 "Mime type rendering requires nbformat>=4.2.0 but it is not installed" 417 ) 419 display_jupyter_version_warnings() 421 ipython_display.display(bundle, raw=True) ValueError: Mime type rendering requires nbformat>=4.2.0 but it is not installed
# Visualize barchart with ranked keywords
topic_model.visualize_barchart()
# Visualize relationships between topics
topic_model.visualize_heatmap(n_clusters=30)
# Visualize the potential hierarchical structure of topics
topic_model.visualize_hierarchy()--------------------------------------------------------------------------- ValueError Traceback (most recent call last) File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/IPython/core/formatters.py:984, in IPythonDisplayFormatter.__call__(self, obj) 982 method = get_real_method(obj, self.print_method) 983 if method is not None: --> 984 method() 985 return True File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/plotly/basedatatypes.py:850, in BaseFigure._ipython_display_(self) 847 import plotly.io as pio 849 if pio.renderers.render_on_display and pio.renderers.default: --> 850 pio.show(self) 851 else: 852 print(repr(self)) File ~/Desktop/0_computation_notes/ml/3_GenerativeModels/AG_LLM/.venv/lib/python3.11/site-packages/plotly/io/_renderers.py:415, in show(fig, renderer, validate, **kwargs) 410 raise ValueError( 411 "Mime type rendering requires ipython but it is not installed" 412 ) 414 if not nbformat or Version(nbformat.__version__) < Version("4.2.0"): --> 415 raise ValueError( 416 "Mime type rendering requires nbformat>=4.2.0 but it is not installed" 417 ) 419 display_jupyter_version_warnings() 421 ipython_display.display(bundle, raw=True) ValueError: Mime type rendering requires nbformat>=4.2.0 but it is not installed
Representation Models
In these examples that follow, we will update our topic representations after having trained our model. This allows for quick iteration. If, however, you want to use a representation model at the start of training, you will need to run it as follows:
from bertopic.representation import KeyBERTInspired
from bertopic import BERTopic
# Create your representation model
representation_model = KeyBERTInspired()
# Use the representation model in BERTopic on top of the default pipeline
topic_model = BERTopic(representation_model=representation_model)To use the representation models, we are first going to duplicate our topic model such that easily show the differences between a model with and without representation model.
# Save original representations
from copy import deepcopy
original_topics = deepcopy(topic_model.topic_representations_)def topic_differences(model, original_topics, nr_topics=5):
"""Show the differences in topic representations between two models """
df = pd.DataFrame(columns=["Topic", "Original", "Updated"])
for topic in range(nr_topics):
# Extract top 5 words per topic per model
og_words = " | ".join(list(zip(*original_topics[topic]))[0][:5])
new_words = " | ".join(list(zip(*model.get_topic(topic)))[0][:5])
df.loc[len(df)] = [topic, og_words, new_words]
return dfKeyBERTInspired
from bertopic.representation import KeyBERTInspired
# Update our topic representations using KeyBERTInspired
representation_model = KeyBERTInspired()
topic_model.update_topics(abstracts, representation_model=representation_model)
# Show topic differences
topic_differences(topic_model, original_topics)| Topic | Original | Updated | |
|---|---|---|---|
| 0 | 0 | speech | asr | recognition | end | acoustic | language | phonetic | speech | encoder | trans... |
| 1 | 1 | translation | nmt | machine | bleu | neural | translation | translate | translations | trans... |
| 2 | 2 | summarization | summaries | summary | abstract... | summarization | summarizers | summaries | summ... |
| 3 | 3 | hate | offensive | speech | detection | toxic | hate | hateful | language | languages | offensive |
| 4 | 4 | relation | extraction | re | relations | entity | relation | relations | relational | extracting... |
Maximal Marginal Relevance
from bertopic.representation import MaximalMarginalRelevance
# Update our topic representations to MaximalMarginalRelevance
representation_model = MaximalMarginalRelevance(diversity=0.5)
topic_model.update_topics(abstracts, representation_model=representation_model)
# Show topic differences
topic_differences(topic_model, original_topics)| Topic | Original | Updated | |
|---|---|---|---|
| 0 | 0 | speech | asr | recognition | end | acoustic | speech | asr | error | model | training |
| 1 | 1 | translation | nmt | machine | bleu | neural | translation | nmt | bleu | parallel | multilin... |
| 2 | 2 | summarization | summaries | summary | abstract... | summarization | extractive | rouge | sentences... |
| 3 | 3 | hate | offensive | speech | detection | toxic | offensive | toxic | hateful | platforms | dataset |
| 4 | 4 | relation | extraction | re | relations | entity | extraction | re | relations | entity | level |
Text Generation
Flan-T5
from transformers import pipeline
from bertopic.representation import TextGeneration
prompt = """I have a topic that contains the following documents:
[DOCUMENTS]
The topic is described by the following keywords: '[KEYWORDS]'.
Based on the documents and keywords, what is this topic about?"""
# Update our topic representations using Flan-T5
generator = pipeline('text2text-generation', model='google/flan-t5-small')
representation_model = TextGeneration(
generator, prompt=prompt, doc_length=50, tokenizer="whitespace"
)
topic_model.update_topics(abstracts, representation_model=representation_model)
# Show topic differences
topic_differences(topic_model, original_topics)Device set to use cuda:0
6%|▌ | 10/164 [00:00<00:10, 14.76it/s]You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset
100%|██████████| 164/164 [00:18<00:00, 8.74it/s]| Topic | Original | Updated | |
|---|---|---|---|
| 0 | 0 | speech | asr | recognition | end | acoustic | Speech-to-sequence modeling | | | | |
| 1 | 1 | translation | nmt | machine | bleu | neural | Science/Tech | | | | |
| 2 | 2 | summarization | summaries | summary | abstract... | Summarization | | | | |
| 3 | 3 | hate | offensive | speech | detection | toxic | hate speech | | | | |
| 4 | 4 | relation | extraction | re | relations | entity | relation extraction | | | | |
OpenAI
import openai
from bertopic.representation import OpenAI
prompt = """
I have a topic that contains the following documents:
[DOCUMENTS]
The topic is described by the following keywords: [KEYWORDS]
Based on the information above, extract a short topic label in the following format:
topic: <short topic label>
"""
# Update our topic representations using GPT-3.5
client = openai.OpenAI(api_key="YOUR_KEY_HERE")
representation_model = OpenAI(
client, model="gpt-3.5-turbo", exponential_backoff=True, chat=True, prompt=prompt
)
topic_model.update_topics(abstracts, representation_model=representation_model)
# Show topic differences
topic_differences(topic_model, original_topics)100%|██████████| 156/156 [02:13<00:00, 1.17it/s] Topic Original \
0 0 speech | asr | recognition | end | acoustic
1 1 medical | clinical | biomedical | patient | he...
2 2 sentiment | aspect | analysis | reviews | opinion
3 3 translation | nmt | machine | neural | bleu
4 4 summarization | summaries | summary | abstract...
Updated
0 Leveraging External Data for Improving Low-Res...
1 Improved Representation Learning for Biomedica...
2 "Advancements in Aspect-Based Sentiment Analys...
3 Neural Machine Translation Enhancements
4 Document Summarization Techniques # Visualize topics and documents
fig = topic_model.visualize_document_datamap(
titles,
topics=list(range(20)),
reduced_embeddings=reduced_embeddings,
width=1200,
label_font_size=11,
label_wrap_width=20,
use_medoids=True,
)
plt.savefig("datamapplot.png", dpi=300)
BONUS: Word Cloud
Make sure to pip install wordcloud first in order to follow this bonus:
First, we need to make sure that each topic is described by a bit more words than just 10 as that would make for a much more interesting wordcloud.
topic_model.update_topics(abstracts, top_n_words=500)Then, we can run the following code to generate the wordcloud for our topic modeling topic:
from wordcloud import WordCloud
import matplotlib.pyplot as plt
def create_wordcloud(model, topic):
plt.figure(figsize=(10,5))
text = {word: value for word, value in model.get_topic(topic)}
wc = WordCloud(background_color="white", max_words=1000, width=1600, height=800)
wc.generate_from_frequencies(text)
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.show()
# Show wordcloud
create_wordcloud(topic_model, topic=17)